Esse notebook R tem como finalidade conhecer e exercitar o conjunto de pacotes tidymodel, que seria uma evolução do pacote caret para transformação e fitting de dados.

Reproduz ou se baseia nos seguintes tutoriais da internet:

Seguindo o exemplo

Bibliotecas

set.seed(42)
options(max.print = 150)

library(modeldata)
library(tidymodels)
library(tidyverse)
library(caret)
library(magrittr)
library(naniar)
library(furrr)
library(skimr)
library(vip)
library(workflows)
library(tune)

plan(multicore)  

Data preparation


data("credit_data")

credit_data <- credit_data %>% set_names(tolower(names(.)))

glimpse(credit_data)
Rows: 4,454
Columns: 14
$ status    <fct> good, good, bad, good, good, good, good, good, good, bad, good, good, good, good, ...
$ seniority <int> 9, 17, 10, 0, 0, 1, 29, 9, 0, 0, 6, 7, 8, 19, 0, 0, 15, 33, 0, 1, 2, 5, 1, 27, 26,...
$ home      <fct> rent, rent, owner, rent, rent, owner, owner, parents, owner, parents, owner, owner...
$ time      <int> 60, 60, 36, 60, 36, 60, 60, 12, 60, 48, 48, 36, 60, 36, 18, 24, 24, 24, 48, 60, 60...
$ age       <int> 30, 58, 46, 24, 26, 36, 44, 27, 32, 41, 34, 29, 30, 37, 21, 68, 52, 68, 36, 31, 25...
$ marital   <fct> married, widow, married, single, single, married, married, single, married, marrie...
$ records   <fct> no, no, yes, no, no, no, no, no, no, no, no, no, no, no, yes, no, no, no, no, no, ...
$ job       <fct> freelance, fixed, freelance, fixed, fixed, fixed, fixed, fixed, freelance, partime...
$ expenses  <int> 73, 48, 90, 63, 46, 75, 75, 35, 90, 90, 60, 60, 75, 75, 35, 75, 35, 65, 45, 35, 46...
$ income    <int> 129, 131, 200, 182, 107, 214, 125, 80, 107, 80, 125, 121, 199, 170, 50, 131, 330, ...
$ assets    <int> 0, 0, 3000, 2500, 0, 3500, 10000, 0, 15000, 0, 4000, 3000, 5000, 3500, 0, 4162, 16...
$ debt      <int> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2500, 260, 0, 0, 0, 2000, 0, 0, 0, 0, 500, 0, ...
$ amount    <int> 800, 1000, 2000, 900, 310, 650, 1600, 200, 1200, 1200, 1150, 650, 1500, 600, 400, ...
$ price     <int> 846, 1658, 2985, 1325, 910, 1645, 1800, 1093, 1957, 1468, 1577, 915, 1650, 940, 50...

Checking missing data com naniar e avalinado


credit_data %>% 
  miss_var_summary()
NA

avaliando os dados


credit_data %>% skim()
-- Data Summary ------------------------
                           Values    
Name                       Piped data
Number of rows             4454      
Number of columns          14        
_______________________              
Column type frequency:               
  factor                   5         
  numeric                  9         
________________________             
Group variables            None      

-- Variable type: factor --------------------------------------------------------------------------------
# A tibble: 5 x 6
  skim_variable n_missing complete_rate ordered n_unique top_counts                              
* <chr>             <int>         <dbl> <lgl>      <int> <chr>                                   
1 status                0         1     FALSE          2 goo: 3200, bad: 1254                    
2 home                  6         0.999 FALSE          6 own: 2107, ren: 973, par: 783, oth: 319 
3 marital               1         1.00  FALSE          5 mar: 3241, sin: 977, sep: 130, wid: 67  
4 records               0         1     FALSE          2 no: 3681, yes: 773                      
5 job                   2         1.00  FALSE          4 fix: 2805, fre: 1024, par: 452, oth: 171

-- Variable type: numeric -------------------------------------------------------------------------------
# A tibble: 9 x 11
  skim_variable n_missing complete_rate    mean       sd    p0   p25   p50   p75   p100 hist 
* <chr>             <int>         <dbl>   <dbl>    <dbl> <dbl> <dbl> <dbl> <dbl>  <dbl> <chr>
1 seniority             0         1        7.99     8.17     0    2      5   12      48 ▇▃▁▁▁
2 time                  0         1       46.4     14.7      6   36     48   60      72 ▁▂▅▃▇
3 age                   0         1       37.1     11.0     18   28     36   45      68 ▆▇▆▃▁
4 expenses              0         1       55.6     19.5     35   35     51   72     180 ▇▃▁▁▁
5 income              381         0.914  142.      80.7      6   90    125  170     959 ▇▂▁▁▁
6 assets               47         0.989 5404.   11574.       0    0   3000 6000  300000 ▇▁▁▁▁
7 debt                 18         0.996  343.    1246.       0    0      0    0   30000 ▇▁▁▁▁
8 amount                0         1     1039.     475.     100  700   1000 1300    5000 ▇▆▁▁▁
9 price                 0         1     1463.     628.     105 1117.  1400 1692.  11140 ▇▁▁▁▁

Class Balance

table(credit_data$status)

 bad good 
1254 3200 
round(prop.table(table(credit_data$status)),2)

 bad good 
0.28 0.72 

modeling


split <- rsample::initial_split(credit_data, prop = .8, strata = "status")

df_train <- training(split)
df_test  <- testing(split)

train_cv <- rsample::vfold_cv(df_train, v=5, strata = "status")
train_cv_caret <- rsample2caret(train_cv)

In this particular example I mainly focus on imputting missing data or assigning them a new categorical level, infrequent/ unobserved values and hot-encoding them.


my_recipe <- df_train %>% 
  recipe(status~.) %>% 
  # imputation: add "unknown" to all missing factor values
  step_unknown(all_nominal(), -status) %>% 
  # imputation: add median to all missing numeric values
  step_medianimpute(all_numeric()) %>% 
  # compining: group factors below 5% of frequency (default) in an "infrequent_combined"
  step_other(all_nominal(), -status, other = "infrequent_combined") %>% 
  # create a "level" in the factor columns for unseen factors
  step_novel(all_nominal(), -status, new_level = "unrecorded_observation") %>% 
  # OHE
  step_dummy(all_nominal(), -status, one_hot=T)

my_recipe
Data Recipe

Inputs:

Operations:

Unknown factor level assignment for all_nominal, -, status
Median Imputation for all_numeric
Collapsing factor levels for all_nominal, -, status
Novel factor level assignment for all_nominal, -, status
Dummy variables from all_nominal, -, status
my_recipe_prep <- prep(my_recipe, retain=T)  

my_recipe_prep
Data Recipe

Inputs:

Training data contained 3565 data points and 335 incomplete rows. 

Operations:

Unknown factor level assignment for home, marital, records, job [trained]
Median Imputation for seniority, time, age, expenses, income, assets, debt, amount, price [trained]
Collapsing factor levels for home, marital, records, job [trained]
Novel factor level assignment for home, marital, records, job [trained]
Dummy variables from home, marital, records, job [trained]
tidy(my_recipe_prep)
NA

playing with recipe

# some data
mydata <- tibble(
  class = as.factor(c("dog","dog","dog","cat","cat","monkey")),
  size  = c(53,42,83,20,30,70),
  weight = c(15,8,12,5,6,21)
)

# create a recipe from "mydata"
rcp <- recipe(mydata) %>% 
  # one hot encoding fo class
  step_dummy(all_nominal(), one_hot = T)

# prepare the recipe
p_rcp <- prep(rcp, verbose = T)
oper 1 step dummy [training] 
The retained training set is ~ 0 Mb  in memory.
# apply a prepared recipe to a new data
# in this case do OHE
bake(p_rcp, mydata)
NA

Caret


control_caret <- trainControl(
  method="cv", 
  verboseIter = F,
  classProbs = T, 
  summaryFunction = twoClassSummary,
  returnResamp = "final",
  savePredictions = "final",
  index = train_cv_caret$index,
  indexOut = train_cv_caret$indexOut
)

grid_caret <- expand.grid(
  mtry = seq(1,ncol(df_train)-1,3),
  splitrule = c("extratrees","gini"),
  min.node.size=c(1,3,5)
)


model_caret <- train(
    status~.,
    data=juice(my_recipe_prep), 
    method="ranger", 
    metric="ROC",
    trControl=control_caret, 
    tuneGrid = grid_caret,
    importance="impurity",
    num.tree=500
  )

print(model_caret)
Random Forest 

3565 samples
  29 predictor
   2 classes: 'bad', 'good' 

No pre-processing
Resampling: Cross-Validated (10 fold) 
Summary of sample sizes: 2851, 2852, 2852, 2852, 2853 
Resampling results across tuning parameters:

  mtry  splitrule   min.node.size  ROC        Sens          Spec     
   1    extratrees  1              0.7903241  0.0000000000  1.0000000
   1    extratrees  3              0.7891645  0.0000000000  1.0000000
   1    extratrees  5              0.7924344  0.0000000000  1.0000000
   1    gini        1              0.8198287  0.0000000000  1.0000000
   1    gini        3              0.8181632  0.0009950249  1.0000000
   1    gini        5              0.8186128  0.0019950249  1.0000000
   4    extratrees  1              0.8063489  0.4432786070  0.9219032
   4    extratrees  3              0.8073462  0.4373084577  0.9187835
   4    extratrees  5              0.8071579  0.4413034826  0.9230750
   4    gini        1              0.8352239  0.4661791045  0.9273696
   4    gini        3              0.8340549  0.4641940299  0.9254173
   4    gini        5              0.8339762  0.4771293532  0.9258087
   7    extratrees  1              0.8096946  0.4741542289  0.9078490
   7    extratrees  3              0.8090484  0.4801293532  0.9086295
   7    extratrees  5              0.8115375  0.4801293532  0.9117530
   7    gini        1              0.8320368  0.4980298507  0.9117553
   7    gini        3              0.8327291  0.4980497512  0.9121459
   7    gini        5              0.8324745  0.4880895522  0.9148788
  10    extratrees  1              0.8092484  0.4860945274  0.9043342
  10    extratrees  3              0.8095218  0.4821194030  0.9062865
  10    extratrees  5              0.8116609  0.4771343284  0.9090186
  10    gini        1              0.8305936  0.5099950249  0.9082427
  10    gini        3              0.8315586  0.5149751244  0.9074607
  10    gini        5              0.8328366  0.5119800995  0.9066787
  13    extratrees  1              0.8077373  0.4900646766  0.9058959
 [ reached getOption("max.print") -- omitted 5 rows ]

ROC was used to select the optimal model using the largest value.
The final values used for the model were mtry = 4, splitrule = gini and min.node.size = 1.
percent(roc_auc(df_test_pred_caret, truth, estimate)$.estimate)
[1] "82%"

tidymodel

print(engine_tidym)
Random Forest Model Specification (classification)

Main Arguments:
  mtry = tune()
  trees = tune()
  min_n = tune()

Engine-Specific Arguments:
  importance = impurity

Computational engine: ranger 
gc()
           used  (Mb) gc trigger  (Mb) max used  (Mb)
Ncells  4713503 251.8    8741388 466.9  8357776 446.4
Vcells 12896293  98.4   23139468 176.6 23139468 176.6
gridy_tidym
#  5-fold cross-validation using stratification 
(wkfl_tidym_best <- finalize_workflow(wkfl_tidym, grid_tidym_best))
== Workflow ====================================================================
Preprocessor: Recipe
Model: rand_forest()

-- Preprocessor ----------------------------------------------------------------
5 Recipe Steps

* step_unknown()
* step_medianimpute()
* step_other()
* step_novel()
* step_dummy()

-- Model -----------------------------------------------------------------------
Random Forest Model Specification (classification)

Main Arguments:
  mtry = 3
  trees = 983
  min_n = 4

Engine-Specific Arguments:
  importance = impurity

Computational engine: ranger 
(wkfl_tidym_final <- last_fit(wkfl_tidym_best, split = split))
! Resample1: model (predictions): Novel levels found in column 'home': NA. The leve...
# Monte Carlo cross-validation (0.8/0.2) with 1 resamples  
# Cross-validated training performance
percent(show_best(gridy_tidym, metric="roc_auc", n = 1)$mean)
[1] "83%"
# Test performance
percent(wkfl_tidym_final$.metrics[[1]]$.estimate[[2]])
[1] "81%"
LS0tDQp0aXRsZTogIkxlYXJuaW5nIFRpZHltb2RlbHMiDQpvdXRwdXQ6IGh0bWxfbm90ZWJvb2sNCi0tLQ0KDQpFc3NlIG5vdGVib29rIFIgdGVtIGNvbW8gZmluYWxpZGFkZSBjb25oZWNlciBlIGV4ZXJjaXRhciBvIGNvbmp1bnRvIGRlIHBhY290ZXMgYHRpZHltb2RlbGAsIHF1ZSBzZXJpYSB1bWEgZXZvbHXDp8OjbyBkbyBwYWNvdGUgYGNhcmV0YCBwYXJhIHRyYW5zZm9ybWHDp8OjbyBlIGZpdHRpbmcgZGUgZGFkb3MuDQoNClJlcHJvZHV6IG91IHNlIGJhc2VpYSBub3Mgc2VndWludGVzIHR1dG9yaWFpcyBkYSBpbnRlcm5ldDoNCg0KKiBbQ2FyZXQgdnMuIHRpZHltb2RlbHMgLSBjb21wYXJpbmcgdGhlIG9sZCBhbmQgbmV3XShodHRwczovL2tvbnJhZHNlbXNjaC5uZXRsaWZ5LmFwcC8yMDE5LzA4L2NhcmV0LXZzLXRpZHltb2RlbHMtY29tcGFyaW5nLXRoZS1vbGQtYW5kLW5ldy8pDQoqIFtJTVBMRU1FTlRJTkcgVEhFIFNVUEVSIExFQVJORVIgV0lUSCBUSURZTU9ERUxTXShodHRwczovL3d3dy5hbGV4cGdoYXllcy5jb20vYmxvZy9pbXBsZW1lbnRpbmctdGhlLXN1cGVyLWxlYXJuZXItd2l0aC10aWR5bW9kZWxzLykNCg0KIyBTZWd1aW5kbyBvIGV4ZW1wbG8NCg0KIyMgQmlibGlvdGVjYXMNCg0KYGBge3IgbG9hZGxpYiwgbWVzc2FnZT1GQUxTRSwgd2FybmluZz1GQUxTRX0NCnNldC5zZWVkKDQyKQ0Kb3B0aW9ucyhtYXgucHJpbnQgPSAxNTApDQoNCmxpYnJhcnkobW9kZWxkYXRhKQ0KbGlicmFyeSh0aWR5bW9kZWxzKQ0KbGlicmFyeSh0aWR5dmVyc2UpDQpsaWJyYXJ5KGNhcmV0KQ0KbGlicmFyeShtYWdyaXR0cikNCmxpYnJhcnkobmFuaWFyKQ0KbGlicmFyeShmdXJycikNCmxpYnJhcnkoc2tpbXIpDQpsaWJyYXJ5KHZpcCkNCmxpYnJhcnkod29ya2Zsb3dzKQ0KbGlicmFyeSh0dW5lKQ0KDQpwbGFuKG11bHRpY29yZSkgIA0KYGBgDQoNCiMjIERhdGEgcHJlcGFyYXRpb24NCg0KYGBge3J9DQoNCmRhdGEoImNyZWRpdF9kYXRhIikNCg0KY3JlZGl0X2RhdGEgPC0gY3JlZGl0X2RhdGEgJT4lIHNldF9uYW1lcyh0b2xvd2VyKG5hbWVzKC4pKSkNCg0KZ2xpbXBzZShjcmVkaXRfZGF0YSkNCg0KYGBgDQoNCiMjIENoZWNraW5nIG1pc3NpbmcgZGF0YSBjb20gYG5hbmlhcmAgZSBhdmFsaW5hZG8gDQoNCmBgYHtyfQ0KDQpjcmVkaXRfZGF0YSAlPiUgDQogIG1pc3NfdmFyX3N1bW1hcnkoKQ0KDQpgYGANCg0KIyMgYXZhbGlhbmRvIG9zIGRhZG9zDQoNCmBgYHtyfQ0KDQpjcmVkaXRfZGF0YSAlPiUgc2tpbSgpDQoNCmBgYA0KDQoNCiMjIENsYXNzIEJhbGFuY2UNCg0KYGBge3J9DQp0YWJsZShjcmVkaXRfZGF0YSRzdGF0dXMpDQpyb3VuZChwcm9wLnRhYmxlKHRhYmxlKGNyZWRpdF9kYXRhJHN0YXR1cykpLDIpDQpgYGANCg0KIyMgbW9kZWxpbmcNCg0KYGBge3J9DQoNCnNwbGl0IDwtIHJzYW1wbGU6OmluaXRpYWxfc3BsaXQoY3JlZGl0X2RhdGEsIHByb3AgPSAuOCwgc3RyYXRhID0gInN0YXR1cyIpDQoNCmRmX3RyYWluIDwtIHRyYWluaW5nKHNwbGl0KQ0KZGZfdGVzdCAgPC0gdGVzdGluZyhzcGxpdCkNCg0KdHJhaW5fY3YgPC0gcnNhbXBsZTo6dmZvbGRfY3YoZGZfdHJhaW4sIHY9NSwgc3RyYXRhID0gInN0YXR1cyIpDQp0cmFpbl9jdl9jYXJldCA8LSByc2FtcGxlMmNhcmV0KHRyYWluX2N2KQ0KDQpgYGANCg0KDQpJbiB0aGlzIHBhcnRpY3VsYXIgZXhhbXBsZSBJIG1haW5seSBmb2N1cyBvbiBpbXB1dHRpbmcgbWlzc2luZyBkYXRhIG9yIGFzc2lnbmluZyB0aGVtIGEgbmV3IGNhdGVnb3JpY2FsIGxldmVsLCBpbmZyZXF1ZW50LyB1bm9ic2VydmVkIHZhbHVlcyBhbmQgaG90LWVuY29kaW5nIHRoZW0uDQoNCg0KYGBge3J9DQoNCm15X3JlY2lwZSA8LSBkZl90cmFpbiAlPiUgDQogIHJlY2lwZShzdGF0dXN+LikgJT4lIA0KICAjIGltcHV0YXRpb246IGFkZCAidW5rbm93biIgdG8gYWxsIG1pc3NpbmcgZmFjdG9yIHZhbHVlcw0KICBzdGVwX3Vua25vd24oYWxsX25vbWluYWwoKSwgLXN0YXR1cykgJT4lIA0KICAjIGltcHV0YXRpb246IGFkZCBtZWRpYW4gdG8gYWxsIG1pc3NpbmcgbnVtZXJpYyB2YWx1ZXMNCiAgc3RlcF9tZWRpYW5pbXB1dGUoYWxsX251bWVyaWMoKSkgJT4lIA0KICAjIGNvbXBpbmluZzogZ3JvdXAgZmFjdG9ycyBiZWxvdyA1JSBvZiBmcmVxdWVuY3kgKGRlZmF1bHQpIGluIGFuICJpbmZyZXF1ZW50X2NvbWJpbmVkIg0KICBzdGVwX290aGVyKGFsbF9ub21pbmFsKCksIC1zdGF0dXMsIG90aGVyID0gImluZnJlcXVlbnRfY29tYmluZWQiKSAlPiUgDQogICMgY3JlYXRlIGEgImxldmVsIiBpbiB0aGUgZmFjdG9yIGNvbHVtbnMgZm9yIHVuc2VlbiBmYWN0b3JzDQogIHN0ZXBfbm92ZWwoYWxsX25vbWluYWwoKSwgLXN0YXR1cywgbmV3X2xldmVsID0gInVucmVjb3JkZWRfb2JzZXJ2YXRpb24iKSAlPiUgDQogICMgT0hFDQogIHN0ZXBfZHVtbXkoYWxsX25vbWluYWwoKSwgLXN0YXR1cywgb25lX2hvdD1UKQ0KDQpteV9yZWNpcGUNCm15X3JlY2lwZV9wcmVwIDwtIHByZXAobXlfcmVjaXBlLCByZXRhaW49VCkgIA0KDQpteV9yZWNpcGVfcHJlcA0KdGlkeShteV9yZWNpcGVfcHJlcCkNCg0KYGBgDQoNCiMjIHBsYXlpbmcgd2l0aCByZWNpcGUNCg0KYGBge3J9DQojIHNvbWUgZGF0YQ0KbXlkYXRhIDwtIHRpYmJsZSgNCiAgY2xhc3MgPSBhcy5mYWN0b3IoYygiZG9nIiwiZG9nIiwiZG9nIiwiY2F0IiwiY2F0IiwibW9ua2V5IikpLA0KICBzaXplICA9IGMoNTMsNDIsODMsMjAsMzAsNzApLA0KICB3ZWlnaHQgPSBjKDE1LDgsMTIsNSw2LDIxKQ0KKQ0KDQojIGNyZWF0ZSBhIHJlY2lwZSBmcm9tICJteWRhdGEiDQpyY3AgPC0gcmVjaXBlKG15ZGF0YSkgJT4lIA0KICAjIG9uZSBob3QgZW5jb2RpbmcgZm8gY2xhc3MNCiAgc3RlcF9kdW1teShhbGxfbm9taW5hbCgpLCBvbmVfaG90ID0gVCkNCg0KIyBwcmVwYXJlIHRoZSByZWNpcGUNCnBfcmNwIDwtIHByZXAocmNwLCB2ZXJib3NlID0gVCkNCg0KIyBhcHBseSBhIHByZXBhcmVkIHJlY2lwZSB0byBhIG5ldyBkYXRhDQojIGluIHRoaXMgY2FzZSBkbyBPSEUNCmJha2UocF9yY3AsIG15ZGF0YSkNCg0KYGBgDQoNCiMjIENhcmV0DQoNCmBgYHtyfQ0KDQpjb250cm9sX2NhcmV0IDwtIHRyYWluQ29udHJvbCgNCiAgbWV0aG9kPSJjdiIsIA0KICB2ZXJib3NlSXRlciA9IEYsDQogIGNsYXNzUHJvYnMgPSBULCANCiAgc3VtbWFyeUZ1bmN0aW9uID0gdHdvQ2xhc3NTdW1tYXJ5LA0KICByZXR1cm5SZXNhbXAgPSAiZmluYWwiLA0KICBzYXZlUHJlZGljdGlvbnMgPSAiZmluYWwiLA0KICBpbmRleCA9IHRyYWluX2N2X2NhcmV0JGluZGV4LA0KICBpbmRleE91dCA9IHRyYWluX2N2X2NhcmV0JGluZGV4T3V0DQopDQoNCmdyaWRfY2FyZXQgPC0gZXhwYW5kLmdyaWQoDQogIG10cnkgPSBzZXEoMSxuY29sKGRmX3RyYWluKS0xLDMpLA0KICBzcGxpdHJ1bGUgPSBjKCJleHRyYXRyZWVzIiwiZ2luaSIpLA0KICBtaW4ubm9kZS5zaXplPWMoMSwzLDUpDQopDQoNCg0KbW9kZWxfY2FyZXQgPC0gdHJhaW4oDQogICAgc3RhdHVzfi4sDQogICAgZGF0YT1qdWljZShteV9yZWNpcGVfcHJlcCksIA0KICAgIG1ldGhvZD0icmFuZ2VyIiwgDQogICAgbWV0cmljPSJST0MiLA0KICAgIHRyQ29udHJvbD1jb250cm9sX2NhcmV0LCANCiAgICB0dW5lR3JpZCA9IGdyaWRfY2FyZXQsDQogICAgaW1wb3J0YW5jZT0iaW1wdXJpdHkiLA0KICAgIG51bS50cmVlPTUwMA0KICApDQoNCnByaW50KG1vZGVsX2NhcmV0KQ0KYGBgDQoNCmBgYHtyfQ0KDQp2YXJJbXAobW9kZWxfY2FyZXQsIHNjYWxlPVQpJGltcG9ydGFuY2UgJT4lIA0KICByb3duYW1lc190b19jb2x1bW4oKSAlPiUgDQogIGFycmFuZ2UoZGVzYyhPdmVyYWxsKSkNCiAgDQpgYGANCg0KYGBge3J9DQoNCmRmX3RyYWluX3ByZWRfY2FyZXQgPC0gbW9kZWxfY2FyZXQkcHJlZCAlPiUgDQogIGdyb3VwX2J5KHJvd0luZGV4KSAlPiUgDQogIHN1bW1hcmlzZShiYWQ9bWVhbihiYWQpKSAlPiUgDQogIHRyYW5zbXV0ZShlc3RpbWF0ZT1iYWQpICU+JSANCiAgYWRkX2NvbHVtbih0cnV0aD1kZl90cmFpbiRzdGF0dXMpDQoNCiMgY3Jvc3MtdmFsaWRhdGVkIHRyYWluaW5nIHBlcmZvcm1hbmNlDQpwZXJjZW50KHJvY19hdWMoZGZfdHJhaW5fcHJlZF9jYXJldCwgdHJ1dGgsIGVzdGltYXRlKSQuZXN0aW1hdGUpDQoNCmBgYA0KYGBge3J9DQojIHRlc3QgcGVyZm9ybWFuY2UNCg0KZGZfdGVzdF9wcmVkX2NhcmV0IDwtIHByZWRpY3QoDQogICAgbW9kZWxfY2FyZXQsIA0KICAgIG5ld2RhdGEgPSBiYWtlKG15X3JlY2lwZV9wcmVwLCBkZl90ZXN0KSwNCiAgICB0eXBlPSJwcm9iIg0KICApICU+JSANCiAgYXNfdGliYmxlKCkgJT4lIA0KICB0cmFuc211dGUoZXN0aW1hdGU9YmFkKSAlPiUgDQogIGFkZF9jb2x1bW4odHJ1dGg9ZGZfdGVzdCRzdGF0dXMpDQoNCnBlcmNlbnQocm9jX2F1YyhkZl90ZXN0X3ByZWRfY2FyZXQsIHRydXRoLCBlc3RpbWF0ZSkkLmVzdGltYXRlKQ0KDQpgYGANCg0KIyB0aWR5bW9kZWwNCg0KYGBge3J9DQplbmdpbmVfdGlkeW0gPC0gcmFuZF9mb3Jlc3QoDQogICAgbW9kZSA9ICJjbGFzc2lmaWNhdGlvbiIsDQogICAgbXRyeSA9IHR1bmUoKSwNCiAgICB0cmVlcyA9IHR1bmUoKSwNCiAgICBtaW5fbiA9IHR1bmUoKQ0KICApICU+JSANCiAgc2V0X2VuZ2luZSgicmFuZ2VyIixpbXBvcnRhbmNlPSJpbXB1cml0eSIpDQoNCnByaW50KGVuZ2luZV90aWR5bSkNCmBgYA0KDQpgYGB7cn0NCmdyaWR0bSA8LSBncmlkX3JhbmRvbSgNCiAgbXRyeSgpICU+JSByYW5nZV9zZXQoYygxLCAyMCkpLA0KICB0cmVlcygpICU+JSByYW5nZV9zZXQoYyg1MDAsIDEwMDApKSwgDQogIG1pbl9uKCkgJT4lIHJhbmdlX3NldChjKDIsIDEwKSksDQogIHNpemUgPSAzMA0KICApDQpgYGANCg0KYGBge3J9DQoNCndrZmxfdGlkeW0gPC0gd29ya2Zsb3coKSAlPiUgDQogIGFkZF9yZWNpcGUobXlfcmVjaXBlKSAlPiUgDQogIGFkZF9tb2RlbChlbmdpbmVfdGlkeW0pDQoNCmdyaWR5X3RpZHltIDwtIHR1bmVfZ3JpZCh3a2ZsX3RpZHltLCByZXNhbXBsZXM9dHJhaW5fY3YsIGdyaWQ9Z3JpZHRtLA0KICAgICAgICAgICAgICAgICAgICAgICAgIG1ldHJpYz1tZXRyaWNfc2V0KHJvY19hdWMpLA0KICAgICAgICAgICAgICAgICAgICAgICAgIGNvbnRyb2w9Y29udHJvbF9ncmlkKHNhdmVfcHJlZD1UKSkNCg0KYGBgDQoNCmBgYHtyfQ0KY29sbGVjdF9tZXRyaWNzKGdyaWR5X3RpZHltKQ0KYGBgDQoNCmBgYHtyfQ0KDQpncmlkX3RpZHltX2Jlc3QgPC0gc2VsZWN0X2Jlc3QoZ3JpZHlfdGlkeW0sIG1ldHJpYz0icm9jX2F1YyIpDQood2tmbF90aWR5bV9iZXN0IDwtIGZpbmFsaXplX3dvcmtmbG93KHdrZmxfdGlkeW0sIGdyaWRfdGlkeW1fYmVzdCkpDQpgYGANCg0KYGBge3J9DQood2tmbF90aWR5bV9maW5hbCA8LSBsYXN0X2ZpdCh3a2ZsX3RpZHltX2Jlc3QsIHNwbGl0ID0gc3BsaXQpKQ0KYGBgDQoNCmBgYHtyfQ0KIyBDcm9zcy12YWxpZGF0ZWQgdHJhaW5pbmcgcGVyZm9ybWFuY2UNCnBlcmNlbnQoc2hvd19iZXN0KGdyaWR5X3RpZHltLCBtZXRyaWM9InJvY19hdWMiLCBuID0gMSkkbWVhbikNCmBgYA0KDQpgYGB7cn0NCiMgVGVzdCBwZXJmb3JtYW5jZQ0KcGVyY2VudCh3a2ZsX3RpZHltX2ZpbmFsJC5tZXRyaWNzW1sxXV0kLmVzdGltYXRlW1syXV0pDQpgYGANCg0KYGBge3J9DQp2aXAocHVsbF93b3JrZmxvd19maXQod2tmbF90aWR5bV9maW5hbCQud29ya2Zsb3dbWzFdXSkpJGRhdGENCmBgYA0KDQo=